package top.timestreet.xmsk.base.boot;

import java.util.List;
import java.util.concurrent.Future;

import org.springframework.scheduling.annotation.AsyncResult;

import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.CollectorPipeline;
import us.codecraft.webmagic.processor.PageProcessor;

/**
* @author TateLiao
* @data 2019年6月2日
* 抓取启动器
*/
public class CrawlerBoot<T> {

	/**
	 * 启动抓取
	 * @param collectorPipeline 处理抓取结果途径
	 * @param url 抓取页面地址
	 * @return
	 */
	@SuppressWarnings({ "rawtypes", "unchecked" })
	protected Future<List<T>> startCrawt(PageProcessor pageProcessor, CollectorPipeline collectorPipeline, String url) {
		Spider.create(pageProcessor).addPipeline(collectorPipeline)
				.setExitWhenComplete(true).addUrl(url).run();
		return new AsyncResult<>(collectorPipeline.getCollected());
	}
	
}
